package com.hspider.test;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;

public class HParser {
	
	public static void extractLinks(String url) {
		HttpURLConnection con = null;
		BufferedInputStream bis = null;
		OutputStream out = null;
		try {
			Parser parser = new Parser(url);
			parser.setEncoding("UTF-8");
			NodeFilter frameFilter = new NodeFilter() {
				@Override
	            public boolean accept(Node node) {
					if(node.getText().startsWith("frame src=")) {
						return true;
	                }
	                return false;
	            }
			};
			NodeFilter imageFilter = new NodeClassFilter(ImageTag.class);
			NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
			NodeList nodeList = parser.extractAllNodesThatMatch(imageFilter);
			for (int i = 0; i < nodeList.size(); i++) {
				Node tag = nodeList.elementAt(i);				
				ImageTag image = (ImageTag)tag;
				String imageURL = image.getImageURL();
				String imageText = image.getText();
				System.out.println("imageURL:"+imageURL);
				System.out.println("imageText:"+imageText);
				con = (HttpURLConnection)(new URL(imageURL).openConnection());
				con.connect();				
				bis = new BufferedInputStream(con.getInputStream());  
				String filename="E://hjk//"+System.currentTimeMillis()+".jpg";
				out = new FileOutputStream(filename);
				byte[] buf = new byte[1024]; 
				int size = 0;				
				while((size = bis.read(buf)) != -1){
					out.write(buf, 0, size);					
				}
			}
		} catch (Exception e) {
			System.err.println(e.getStackTrace());
		} finally {
			try {
				out.close();
				bis.close();
				con.disconnect();
			}
			catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
	// 获取一个网页上所有的链接和图片链接
	public static void extracLinks(String url) {
		try {
			Parser parser = new Parser(url);
			parser.setEncoding("gb2312");
			//过滤 <frame> 标签的 filter，用来提取 frame 标签里的 src 属性所、表示的链接
			NodeFilter frameFilter = new NodeFilter() {
				public boolean accept(Node node) {
					if (node.getText().startsWith("frame src=")) {
						return true;
					} else {
						return false;
					}
				}
			};
			//OrFilter 来设置过滤 <a> 标签，<img> 标签和 <frame> 标签，三个标签是 or 的关系
			OrFilter rorFilter = new OrFilter(new NodeClassFilter(LinkTag.class), new NodeClassFilter(ImageTag.class));
			OrFilter linkFilter = new OrFilter(rorFilter, frameFilter);
			//得到所有经过过滤的标签
			NodeList list = parser.extractAllNodesThatMatch(linkFilter);
			for (int i = 0; i < list.size(); i++) {
				Node tag = list.elementAt(i);
				if (tag instanceof LinkTag)//<a> 标签 
				{
					LinkTag link = (LinkTag) tag;
					String linkUrl = link.getLink();//url
					String text = link.getLinkText();//链接文字
					System.out.println(linkUrl + "**********" + text);
				}
				else if (tag instanceof ImageTag)//<img> 标签
				{
					ImageTag image = (ImageTag) list.elementAt(i);
					System.out.print(image.getImageURL() + "********");//图片地址
					System.out.println(image.getText());//图片文字
				}
				else//<frame> 标签
				{
					//提取 frame 里 src 属性的链接如 <frame src="test.html"/>
					String frame = tag.getText();
					int start = frame.indexOf("src=");
					frame = frame.substring(start);
					int end = frame.indexOf(" ");
					if (end == -1)
						end = frame.indexOf(">");
					frame = frame.substring(5, end - 1);
					System.out.println(frame);
				}
			}
		} catch (ParserException e) {
			e.printStackTrace();
		}
	}

}
